{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000\n",
      "20000\n",
      "30000\n",
      "40000\n",
      "50000\n",
      "60000\n",
      "70000\n",
      "80000\n",
      "90000\n",
      "100000\n",
      "110000\n",
      "115672\n"
     ]
    }
   ],
   "source": [
    "import re, os, sys, json, random\n",
    "from tqdm import *\n",
    "import numpy as np\n",
    "import tensorflow as tf  \n",
    "import h5py\n",
    "import nltk\n",
    "\n",
    "\n",
    "def load_word(path):\n",
    "        \n",
    "    input_file = open(path)\n",
    "    word_lst = [line.rstrip('\\n') for line in input_file.readlines()]\n",
    "    words = dict((word, i) for i, word in enumerate(word_lst))\n",
    "    rwords = dict(map(lambda t:(t[1],t[0]), words.items()))\n",
    "    input_file.close()\n",
    "    \n",
    "    return words, rwords\n",
    "\n",
    "\n",
    "def h5py_write(text_path, words, h5_path, source_len, oseq_len, simplified_len, nb_samples):\n",
    "    \n",
    "    eos = words['eos']\n",
    "    pad = words['pad']\n",
    "    go = words['go']\n",
    "    \n",
    "    input_ = open(text_path)\n",
    "    output_ = h5py.File(h5_path, 'a')\n",
    "    \n",
    "    source_set = output_.create_dataset(\"source\", (nb_samples, source_len), dtype='int32')\n",
    "    defendant_set = output_.create_dataset(\"defendant\", (nb_samples, simplified_len), dtype='int32')\n",
    "    label_set = output_.create_dataset(\"label\", (nb_samples, oseq_len), dtype='int32')\n",
    "    ground_truth_set = output_.create_dataset(\"ground_truth\", (nb_samples, oseq_len), dtype='int32')\n",
    "    defendant_length_set = output_.create_dataset(\"defendant_length\", (nb_samples, ), dtype='int32')\n",
    "    weights_set = output_.create_dataset(\"loss_weights\", (nb_samples, oseq_len), dtype='float32')\n",
    "    \n",
    "    \n",
    "    source_tx_set = output_.create_dataset('source_tx', (nb_samples, ), dtype='S3000')\n",
    "    defendant_tx_set = output_.create_dataset('defendant_tx', (nb_samples, ), dtype='S600')\n",
    "    reason_tx_set = output_.create_dataset('reason_tx', (nb_samples, ), dtype='S500')\n",
    "    \n",
    "    sindex = 0\n",
    "    \n",
    "    while True:\n",
    "\n",
    "        texts = []\n",
    "        for i in range(10000):\n",
    "            \n",
    "            text = input_.readline()\n",
    "            if text == '': \n",
    "                to_break = True\n",
    "                break\n",
    "            texts.append(json.loads(text))\n",
    "        \n",
    "            \n",
    "            \n",
    "        source_tx = [[pad for j in range(source_len)] for i in range(len(texts))]\n",
    "        for i in range(len(texts)):\n",
    "            tx = texts[i]['source'][:min(source_len, len(texts[i]['source']))]\n",
    "            start = source_len - len(tx)\n",
    "            for j in range(start, source_len):\n",
    "                source_tx[i][j] = words[tx[j-start]]\n",
    "        source_set[sindex:sindex+len(texts)] = source_tx\n",
    "\n",
    "\n",
    "\n",
    "        defendant_tx = [[pad for j in range(simplified_len)] for i in range(len(texts))]\n",
    "        defendant_length = [min(simplified_len, len(texts[i]['defendant'])) for i in range(len(texts))]\n",
    "        for i in range(len(texts)):\n",
    "            tx = texts[i]['defendant'][:min(simplified_len, len(texts[i]['defendant']))]\n",
    "            start = simplified_len - len(tx)\n",
    "            for j in range(start, simplified_len):\n",
    "                defendant_tx[i][j] = words[tx[j-start]]\n",
    "        defendant_length_set[sindex:sindex+len(texts)] = defendant_length\n",
    "        defendant_set[sindex:sindex+len(texts)] = defendant_tx\n",
    "\n",
    "\n",
    "        \n",
    "        reason_tx = [[pad for j in range(oseq_len)] for i in range(len(texts))]\n",
    "        truth_tx = [[pad for j in range(oseq_len)] for i in range(len(texts))]\n",
    "        lengths = [len(texts[i]['reason']) for i in range(len(texts))]\n",
    "        weigths = [[.0 for j in range(oseq_len)] for i in range(len(texts))]\n",
    "\n",
    "\n",
    "\n",
    "        for i in range(len(texts)):\n",
    "            \n",
    "            tx = texts[i]['reason'][:min(oseq_len, len(texts[i]['reason']))]\n",
    "            if re.search(r'构成', tx): weigth = 1.\n",
    "            else: weigth = 1.\n",
    "            \n",
    "            for j in range(len(tx)):\n",
    "                reason_tx[i][j] = words[tx[j]]\n",
    "                truth_tx[i][j] = words[tx[j]]\n",
    "                weigths[i][j] = weigth\n",
    "            \n",
    "            if len(tx) < oseq_len:\n",
    "                reason_tx[i][len(tx)] = eos\n",
    "                weigths[i][len(tx)] = weigth\n",
    "            truth_tx[i].pop()\n",
    "            truth_tx[i].insert(0, go)\n",
    "\n",
    "        label_set[sindex:sindex+len(texts)] = reason_tx\n",
    "        ground_truth_set[sindex:sindex+len(texts)] = truth_tx\n",
    "        weights_set[sindex:sindex+len(texts)] = weigths\n",
    "        \n",
    "        \n",
    "        encoder_source = [texts[i]['source'].encode('gb2312', 'ignore') for i in range(len(texts))]\n",
    "        encoder_defendant = [texts[i]['defendant'].encode('gb2312', 'ignore') for i in range(len(texts))]\n",
    "        encoder_reason = [texts[i]['reason'].encode('gb2312', 'ignore') for i in range(len(texts))]\n",
    "        \n",
    "        source_tx_set[sindex:sindex+len(texts)] = encoder_source[:]\n",
    "        defendant_tx_set[sindex:sindex+len(texts)] = encoder_defendant[:]\n",
    "        reason_tx_set[sindex:sindex+len(texts)] = encoder_reason[:]\n",
    "        \n",
    "        sindex += len(texts)\n",
    "        \n",
    "        print (sindex)\n",
    "        \n",
    "        if sindex >= nb_samples-1: break\n",
    "        \n",
    "    output_.close()\n",
    "    input_.close()\n",
    "\n",
    "\n",
    "    \n",
    "if __name__ == '__main__':\n",
    "    \n",
    "    \n",
    "    words, rwords = load_word('/home/xuwenshen/data/big_data/2017_3_13/words')\n",
    "    \n",
    "#     h5py_write(text_path='/home/xuwenshen/data/big_data/2017_3_13/shuffled_train',\n",
    "#                words=words, \n",
    "#                h5_path='/home/xuwenshen/data/big_data/2017_3_13/train.h5', \n",
    "#                source_len=1000, \n",
    "#                simplified_len=150,\n",
    "#                oseq_len=200,\n",
    "#                nb_samples=1600000)\n",
    "\n",
    "    h5py_write(text_path='/home/xuwenshen/data/big_data/2017_3_13/shuffled_test',\n",
    "               words=words, \n",
    "               h5_path='/home/xuwenshen/data/big_data/2017_3_13/test.h5', \n",
    "               source_len=1000, \n",
    "               simplified_len=150,\n",
    "               oseq_len=200,\n",
    "               nb_samples=115672)\n",
    "    \n",
    "#     h5py_write(text_path='/home/xuwenshen/data/big_data/2017_3_13/shuffled_valid',\n",
    "#            words=words, \n",
    "#            h5_path='/home/xuwenshen/data/big_data/2017_3_13/valid.h5', \n",
    "#            source_len=1000, \n",
    "#            simplified_len=150,\n",
    "#            oseq_len=200,\n",
    "#            nb_samples=10000)\n",
    "    \n",
    "    \n",
    "    file = h5py.File('/home/xuwenshen/data/big_data/2017_3_13/test.h5')\n",
    "    \n",
    "    for i in range(2000):\n",
    "        source = file['source'][i]\n",
    "        ground_truth = file['ground_truth'][i]\n",
    "        label = file['label'][i]\n",
    "        loss_weights = file['loss_weights'][i]\n",
    "        defendant = file['defendant'][i]\n",
    "\n",
    "        souce_tx = ''\n",
    "        label_tx = ''\n",
    "        ground_truth_tx = ''\n",
    "        defendant_tx = ''\n",
    "        counter = 0\n",
    "\n",
    "        for j in range(len(defendant)):\n",
    "            defendant_tx += rwords[defendant[j]]\n",
    "            \n",
    "        for j in range(len(source)):\n",
    "            souce_tx += rwords[source[j]]\n",
    "            \n",
    "\n",
    "        for j in range(len(label)):\n",
    "            if rwords[label[j]] != 'pad':\n",
    "                counter += 1\n",
    "            label_tx += rwords[label[j]]\n",
    "\n",
    "        for j in range(len(ground_truth)):\n",
    "            ground_truth_tx += rwords[ground_truth[j]]\n",
    "\n",
    "#         print (souce_tx)\n",
    "#         print ('*'*20 + '\\n')\n",
    "#         print (defendant_tx)\n",
    "#         print ('*' * 20 + '\\n')\n",
    "#         print (label_tx)\n",
    "#         print ('*'*20 + '\\n')\n",
    "#         print (ground_truth_tx)\n",
    "#         print ('*'*20 + '\\n')\n",
    "#         print (sum(loss_weights))\n",
    "#         print (counter)\n",
    "#         print ('*'*20 + '\\n')\n",
    "\n",
    "#         print (file['source_tx'][i].decode('gb2312'))\n",
    "#         print ('*' * 20 + '\\n')\n",
    "#         print (file['defendant_tx'][i].decode('gb2312'))\n",
    "#         print ('*' * 20 + '\\n')\n",
    "#         print (file['reason_tx'][i].decode('gb2312'))\n",
    "#         print ('\\n')\n",
    "    file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
